import asyncio
import aiohttp
import json
from lxml import etree
from motor.motor_asyncio import AsyncIOMotorClient
import random
import time

async def fetch(session, url, data_json=None):
    if data_json:
        async with session.post(url, json=data_json) as response:
            return await response.text()
    else:
        async with session.get(url) as response:
            return await response.text()

async def parse_book(session, name, bookId, client):
    book_url = f"https://huayu.zongheng.com/book/{bookId}.html"
    book_html = await fetch(session, book_url)
    root = etree.HTML(book_html)

    nums_divs = root.xpath('//div[@class="nums"]')
    book_dec_divs = root.xpath('//div[@class="book-dec Jbook-dec hide"]')
    all_catalog_btns = root.xpath('//a[@class="all-catalog"]')



    for nums_div, book_dec_div, all_catalog_btn in zip(nums_divs, book_dec_divs, all_catalog_btns):
        word_count = nums_div.xpath('.//span[contains(text(), "字数")]/i/text()')[0].strip()
        total_recommend = nums_div.xpath('.//span[contains(text(), "总推荐")]/i/text()')[0].strip()
        total_click = nums_div.xpath('.//span[contains(text(), "总点击")]/i/text()')[0].strip()
        weekly_recommend = nums_div.xpath('.//span[contains(text(), "周推荐")]/i/text()')[0].strip()
        text_content = ''.join(book_dec_div.xpath('.//p//text()')).strip()
        href = all_catalog_btn.attrib['href']

        book_data = {
            "小说名称": name,
            "字数": word_count,
            "总推荐": total_recommend,
            "总点击": total_click,
            "周推荐": weekly_recommend,
            "简介": text_content,
            "章节": []
        }

        chapter_res = await fetch(session, href)
        chapter_html = etree.HTML(chapter_res)
        links = chapter_html.xpath('//li[@class=" col-4"]/a')

        for link in links:
            text = link.text
            href = link.get('href')
            await asyncio.sleep(random.uniform(1, 3))

            story_res = await fetch(session, href)
            story_html = etree.HTML(story_res)
            content_divs = story_html.xpath('//div[@class="content"]')
            if content_divs:
                content_div = content_divs[0]
                paragraphs = content_div.xpath('.//p')
                story_content = '\n'.join([p.text.strip() for p in paragraphs if p.text])
            else:
                story_content = "未找到章节内容"

            chapter_data = {
                "章节名称": text,
                "章节内容": story_content
            }
            book_data["章节"].append(chapter_data)

        collection = client["纵横小说"]["女生榜"]
        await collection.insert_one(book_data)

async def main():
    client = AsyncIOMotorClient('mongodb://localhost:27017/')  # 请根据实际情况修改 MongoDB 连接字符串
    async with aiohttp.ClientSession() as session:
        page = 1
        while True:
            url = "https://www.zongheng.com/api2/catefine/storeSearch"
            data_json = {
                'worksTypes': 0,
                'bookType': 1,
               'subWorksTypes': 0,
                'totalWord': 0,
               'serialStatus': 1,
                'vip': 9,
                'order': 'weekOrder',
                'pageNum': page,
                'pageSize': 20,
                'categoryId': 0,
                'categoryPid': 0,
                'naodongFilter': 0,
            }

            response_text = await fetch(session, url, data_json)
            html = json.loads(response_text)
            if not html:
                break

            tasks = []
            for i in range(len(html["result"]["bookList"])):
                name = html["result"]["bookList"][i]['name']
                bookId = html["result"]["bookList"][i]['bookId']
                task = asyncio.create_task(parse_book(session, name, bookId, client))
                tasks.append(task)

            await asyncio.gather(*tasks)
            page += 1

if __name__ == "__main__":
    asyncio.run(main())